{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.87057567, 0.49203452, 0.06654195], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAk0klEQVR4nO3df3DU9YH/8ddn82MhJLv5AcmSkki+g1fM8KPKz63WWkmJXqxY6Y1nOcuXUjtywRFpncqd4ujcDI793tl6Z9G7zhXHnuJgRSsH9WKAaCUChh8NiFErmghsIsTsJkB+7vv7B2XPRdAkfJJ9J3k+ZnbGfD7vfee9H2Gf7O4nnzjGGCMAACzkSfQCAAC4ECIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALBWwiL1+OOPa+LEiRo1apTmzJmjXbt2JWopAABLJSRSzz33nFauXKkHHnhAe/bs0fTp01VaWqqmpqZELAcAYCknEReYnTNnjmbNmqV/+7d/kyRFo1EVFBTozjvv1L333jvYywEAWCp5sL9hZ2enampqtGrVqtg2j8ejkpISVVdXn/c+HR0d6ujoiH0djUbV3NysnJwcOY4z4GsGALjLGKPW1lbl5+fL47nwm3qDHqnjx4+rp6dHeXl5cdvz8vL0zjvvnPc+a9as0YMPPjgYywMADKKGhgZNmDDhgvsHPVL9sWrVKq1cuTL2dTgcVmFhoRoaGuTz+RK4MgBAf0QiERUUFCgjI+MLxw16pMaOHaukpCQ1NjbGbW9sbFQgEDjvfbxer7xe7+e2+3w+IgUAQ9iXfWQz6Gf3paamasaMGaqsrIxti0ajqqysVDAYHOzlAAAslpC3+1auXKnFixdr5syZmj17tn7xi1/o5MmTWrJkSSKWAwCwVEIidcstt+iTTz7R6tWrFQqF9LWvfU1/+MMfPncyBQBgZEvIz0ldrEgkIr/fr3A4zGdSADAE9fZ5nGv3AQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALBWnyP12muv6Tvf+Y7y8/PlOI5efPHFuP3GGK1evVrjx4/X6NGjVVJSovfeey9uTHNzsxYtWiSfz6fMzEwtXbpUbW1tF/VAAADDT58jdfLkSU2fPl2PP/74efc/8sgjeuyxx/TEE09o586dGjNmjEpLS9Xe3h4bs2jRIh08eFAVFRXatGmTXnvtNf34xz/u/6MAAAxP5iJIMhs3box9HY1GTSAQMD//+c9j21paWozX6zXPPvusMcaYt99+20gyu3fvjo3ZsmWLcRzHHDlypFffNxwOG0kmHA5fzPIBAAnS2+dxVz+TOnz4sEKhkEpKSmLb/H6/5syZo+rqaklSdXW1MjMzNXPmzNiYkpISeTwe7dy587zzdnR0KBKJxN0AAMOfq5EKhUKSpLy8vLjteXl5sX2hUEi5ublx+5OTk5WdnR0bc641a9bI7/fHbgUFBW4uGwBgqSFxdt+qVasUDodjt4aGhkQvCQAwCFyNVCAQkCQ1NjbGbW9sbIztCwQCampqitvf3d2t5ubm2Jhzeb1e+Xy+uBsAYPhzNVJFRUUKBAKqrKyMbYtEItq5c6eCwaAkKRgMqqWlRTU1NbExW7duVTQa1Zw5c9xcDgBgiEvu6x3a2tr0/vvvx74+fPiw9u3bp+zsbBUWFmrFihX6p3/6J1166aUqKirS/fffr/z8fN10002SpMsuu0zXXXedbr/9dj3xxBPq6urS8uXL9bd/+7fKz8937YEBAIaBvp42uG3bNiPpc7fFixcbY86chn7//febvLw84/V6zbx580xdXV3cHCdOnDC33nqrSU9PNz6fzyxZssS0tra6fuoiAMBOvX0ed4wxJoGN7JdIJCK/369wOMznUwAwBPX2eXxInN0HABiZiBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWCs50QsABkNPe7s6jhxReyiknpMn5TiOkjMy5B0/Xt78fHlSUhK9RADnQaQwrBljdPrwYTX+/vc69f776jx+XNHTpyXHUdKYMUodO1bpxcXKu+kmpebmynGcRC8ZwGcQKQxb0Y4ONb/+uo48/bS6W1okY/53pzHqaW3V6dZWnf7oI7Xu36/8xYuVOWOGnGT+WgC24G8jhqVod7eOv/qqjq1fr+5w+EsGR9X+8cf6+D/+QzJGmXPm8IoKsAQnTmBYOv3nP+vYc899eaA+o7OpSUeeekqdTU0DuDIAfUGkMOxEOzv14WOPnXmLr486jhxRw7//u6Kdne4vDECfESkMO5++8YY6jx/v9/1Pvf++WmtrXVwRgP4iUhh22g4dOnMGXz91ffqpInv3KtrR4eKqAPQHkQLOo2nTJp36859lPntGIIBBR6SA84lGFfrd7xK9CmDEI1LABYT37NHRp5/m1RSQQEQKw443EJDjxmWOenrUWlurjmPHLn4uAP1CpDDsjJ0/XylZWa7MdbKuTp9WV8v09LgyH4C+IVIYdpLT05V11VWuzdf4wgsXdUo7gP4jUhiW8m6+Wd6vfMWVuXpaW9W0aROfTQEJQKQwLCWPGaNx113n2nzN27fr9IcfujYfgN4hUhiWnKQkZV99tXyXX+7KfN2RiBo3blTPqVOuzAegd4gUhq3kzExlBoNyUlMvfjJj1Lx9u5pff10mGr34+QD0CpHCsOU4jsbOn6+sr3/dtTk/2byZV1PAICJSGN4cR4G/+RslpaW5Mt3pw4fV8uabrswF4MsRKQxrjuPIGwhoXFmZa3MeW7+eU9KBQUKkMOx5UlKUddVVGn3JJa7M19XcrBNbt3JKOjAIiBRGhNETJypj+nTJc/F/5E13t46tX6/wrl2EChhgRAojguM4+sptt7n22ZTp7taJbdtkurpcmQ/A+REpjBhOaqoKli6VHMeV+Vqqq3Xqgw9cmQvA+REpjBiO4ygzGFTOtde6M6ExanjySfVcxG8BBvDFiBRGlKS0NGV94xtKyclxZb72Y8f06Y4drswF4POIFEacjKlTlVZU5Mpc0VOndPTpp3Xyvfc4iQIYAEQKI44nJUWX3HmnPF6vK/N1NTcrvGuXxOWSANcRKYxIyX6/xl1/vWvzhX73O37AFxgARAojkuPxKHfBAqVPnerKfKa7Wx+vW8dv8AVcRqQwYqVkZyvnW9+SZ/RoV+Y7eeiQIvv3uzIXgDOIFEass6ekj5owwZX5upqb1bx9u7rb2lyZDwCRwgiXPGaMCn70IzlJSa7M9+mOHWo7cIAz/QCXECmMeGmTJp25rp8LTGenDv/zP6uHV1OAK4gURjwnOVlf+b//17W3/aJdXWp86SVeTQEuIFIY8RzH0eiCAmVddZXkxtt+0aha3nxT7fX1Fz8XMMIRKUCSk5SksaWlGjV+vCvztdfX63hlpaKdna7MB4xURAr4i9ScHOXeeKNrV0n/tKpKHY2NvO0HXAQiBXxG1je+odSxY12Zq+vTT/X+gw/KdHe7Mh8wEhEp4DOS0tI0YelS1+bramnRp2+84dp8wEhDpIDPcBxH6cXF8s+Z48p8prNTxysq1NXS4sp8wEjTp0itWbNGs2bNUkZGhnJzc3XTTTeprq4ubkx7e7vKy8uVk5Oj9PR0LVy4UI2NjXFj6uvrVVZWprS0NOXm5uqee+5RN2+JwBIpmZnK//73XfudU20HD6p5+3au6wf0Q58iVVVVpfLycr355puqqKhQV1eX5s+fr5MnT8bG3H333Xr55Ze1YcMGVVVV6ejRo7r55ptj+3t6elRWVqbOzk7t2LFDTz31lNatW6fVq1e796iAi5RWVKSsK690Z7JoVCcqK9UVDrszHzCCOOYiTj365JNPlJubq6qqKl199dUKh8MaN26cnnnmGX3ve9+TJL3zzju67LLLVF1drblz52rLli264YYbdPToUeXl5UmSnnjiCf3sZz/TJ598otTU1C/9vpFIRH6/X+FwWD6fr7/LB75QR2OjDi5b5tqJD74rrtCkBx6Q49LZg8BQ1tvn8Yv6TCr8l38ZZmdnS5JqamrU1dWlkpKS2JjJkyersLBQ1dXVkqTq6mpNnTo1FihJKi0tVSQS0cGDB8/7fTo6OhSJROJuwEBLHTdOE370I9fmO/nuuzr53nuuzQeMBP2OVDQa1YoVK3TllVdqypQpkqRQKKTU1FRlZmbGjc3Ly1MoFIqN+Wygzu4/u+981qxZI7/fH7sVFBT0d9lArzkejzKmTFHapEmuzNfT1qbG559Xz+nTrswHjAT9jlR5ebkOHDig9evXu7me81q1apXC4XDs1tDQMODfE5Ck0YWFGn/LLUpKS3Nlvsj+/Yrs2cMP+AK91K9ILV++XJs2bdK2bds04TMX5QwEAurs7FTLOafbNjY2KhAIxMace7bf2a/PjjmX1+uVz+eLuwGDxT97tsZMnuzKXNHTp3W8okLRU6dcmQ8Y7voUKWOMli9fro0bN2rr1q0qKiqK2z9jxgylpKSosrIytq2urk719fUKBoOSpGAwqNraWjU1NcXGVFRUyOfzqbi4+GIeCzAgHMfR+O9/37X5Inv26NiGDa7NBwxnfYpUeXm5fvvb3+qZZ55RRkaGQqGQQqGQTv/lPXa/36+lS5dq5cqV2rZtm2pqarRkyRIFg0HNnTtXkjR//nwVFxfrtttu0/79+/XKK6/ovvvuU3l5ubxer/uPEHBB2sSJyl2wwLX5Pn39dU5JB3qhT5Fau3atwuGwrrnmGo0fPz52e+6552JjHn30Ud1www1auHChrr76agUCAb3wwgux/UlJSdq0aZOSkpIUDAb1d3/3d/rBD36ghx56yL1HBbjMSUmRf9YspY4b58p8nZ98oiPr1inKD7EDX+iifk4qUfg5KSSCMUZNL7+sI089JdPVddHzpebmquinP1W6S593AUPJoPycFDCSOI6jcaWl7r2aampSc1WVoi4EDxiuiBTQB05qqiYsWeLafJ9s3qzThw+7Nh8w3BApoA8cx1H6ZZfJP3u2OxMao2MbNnDxWeACiBTQR0kZGcoMBuVx6WzU8O7dOvrsszLRqCvzAcMJkQL6yHEc5VxzjcaWlkoeF/4KRaNq3bdPHRe4LBgwkhEpoB+cpCTlLVggz6hRrsx38t13Famp4dUUcA4iBfRTytixyr/1VtfmO/L00+pqbnZtPmA4IFJAPzmOo8y5czX6nMuD9Ve0o0ONL77oylzAcEGkgIuQOnasMoNByY1fZGiMjr/6qo5XVnKVdOAviBRwEc5+NuXaVdJPnVL4rbfU09rqynzAUEekgIuUNHq0xn7721JSkivztbzxhiK1tbyaAkSkAFfkfOtbyrn2WtfmO/rb38p0dro2HzBUESnADR6Pxl1/vVKyslyZruPYMR1/9VVX5gKGMiIFuMBxHI0uLFTWVVe5M2E0qtDvfqfI/v3uzAcMUUQKcIknNVXjyso06itfcWW+ruPHFdmzh6ukY0QjUoCLvOPHn7n4rBunpEtq3LhRJ995h5MoMGIRKcBFjuMof9EipU+Z4tqcjS++KBEpjFBECnCZk5KivAUL5KSkuDJfZP9+tdbWujIXMNQQKcBljuMoY8oUZV15pSvzmc5OHXvuOXVHIq7MBwwlRAoYAElpacq59lolZ2a6Mt+pDz5QuKaGz6Yw4hApYIBkTJum0RMnujJX9NQpNTz5pNo//phQYUQhUsAAcTweFf30p/KOH+/KfD2nTulEZaUrcwFDBZECBlDymDHKveEG1+b7ZPNmtezaxaspjBhEChhATlKSMoNBZXzta67MF21v19Gnn1bboUOECiMCkQIGWOrYscr+xjd6/avmj5w8qU0NDXr2gw/06tGjOnnOFSfa6+t17Nln1dPWNhDLBaySnOgFACNB9tVXK/T88+o4duyCY4wxOtzWpgf27tWHbW1q7+mRLyVFU7Ky9P9mzVKK53//Tdm6f79O/fnPypg+XY5LV7cAbMQrKWAQeLxe/Z+f/ewLx3zQ1qbb33hDh8Jhne7pkZEU7urSG01NumvnTp1ob48bX//EEwO4YsAORAoYJN5A4Mx1/S7gFwcPKnyBi8nuOn5cFUePxm0zXHgWIwCRAgZJUlqa8r//fY265JJELwUYMogUMIhGFxUpc84c137VPDDcESlgEDmOo9wbb1RyRsbn9pUVFCjlAidBTExP17Ts7IFeHmAdIgUMsuSMDE1YsuRz20vz8/XA5ZdrVFJS7C9mkuMox+vVP8+apeJzrgOYt3DhwC8WSDBOQQcGmeM4ypg+XaMvuUSnP/oobntpfr4mpKVp08cf60R7uyamp+uWoiLleL1xc3jHj1dWMMjp5xj2iBSQAKnZ2Sr6yU8Ueuklfbp9u0xPj6QzoZqSlaUpWVkXvG9yVpbyb7tNyT7fYC0XSBje7gMSZPTEiSr40Y+Ufe21cpJ79+/FpPR0jb/lFmXOni2Hky8wAvBKCkigpLQ0TfjhD5WSna3mbdvU2dR03nFOUpJGFRQo77vfVfY11/A2H0YMIgUkkOM4Sh4zRuO/9z35pk3Tpzt2qO3gQXWEQop2dCgpPV2jJkyQf+ZM+WfO1OjCQgKFEYVIARbweL1KnzJFY/7qr9Rz+rRMd7dMNConKUmelBR50tLk6eVbgsBwwp96wBKO48jxeuU550w+YCTjxAkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABr9SlSa9eu1bRp0+Tz+eTz+RQMBrVly5bY/vb2dpWXlysnJ0fp6elauHChGhsb4+aor69XWVmZ0tLSlJubq3vuuUfd3d3uPBoAwLDSp0hNmDBBDz/8sGpqavTWW2/p2muv1YIFC3Tw4EFJ0t13362XX35ZGzZsUFVVlY4ePaqbb745dv+enh6VlZWps7NTO3bs0FNPPaV169Zp9erV7j4qAMDwYC5SVlaW+fWvf21aWlpMSkqK2bBhQ2zfoUOHjCRTXV1tjDFm8+bNxuPxmFAoFBuzdu1a4/P5TEdHR6+/ZzgcNpJMOBy+2OUDABKgt8/j/f5MqqenR+vXr9fJkycVDAZVU1Ojrq4ulZSUxMZMnjxZhYWFqq6uliRVV1dr6tSpysvLi40pLS1VJBKJvRo7n46ODkUikbgbAGD463OkamtrlZ6eLq/XqzvuuEMbN25UcXGxQqGQUlNTlZmZGTc+Ly9PoVBIkhQKheICdXb/2X0XsmbNGvn9/titoKCgr8sGAAxBfY7UV7/6Ve3bt087d+7UsmXLtHjxYr399tsDsbaYVatWKRwOx24NDQ0D+v0AAHZI7usdUlNTNWnSJEnSjBkztHv3bv3yl7/ULbfcos7OTrW0tMS9mmpsbFQgEJAkBQIB7dq1K26+s2f/nR1zPl6vV16vt69LBQAMcRf9c1LRaFQdHR2aMWOGUlJSVFlZGdtXV1en+vp6BYNBSVIwGFRtba2amppiYyoqKuTz+VRcXHyxSwEADDN9eiW1atUqXX/99SosLFRra6ueeeYZbd++Xa+88or8fr+WLl2qlStXKjs7Wz6fT3feeaeCwaDmzp0rSZo/f76Ki4t122236ZFHHlEoFNJ9992n8vJyXikBAD6nT5FqamrSD37wAx07dkx+v1/Tpk3TK6+8om9/+9uSpEcffVQej0cLFy5UR0eHSktL9atf/Sp2/6SkJG3atEnLli1TMBjUmDFjtHjxYj300EPuPioAwLDgGGNMohfRV5FIRH6/X+FwWD6fL9HLAQD0UW+fx7l2HwDAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrXVSkHn74YTmOoxUrVsS2tbe3q7y8XDk5OUpPT9fChQvV2NgYd7/6+nqVlZUpLS1Nubm5uueee9Td3X0xSwEADEP9jtTu3bv15JNPatq0aXHb7777br388svasGGDqqqqdPToUd18882x/T09PSorK1NnZ6d27Nihp556SuvWrdPq1av7/ygAAMOT6YfW1lZz6aWXmoqKCvPNb37T3HXXXcYYY1paWkxKSorZsGFDbOyhQ4eMJFNdXW2MMWbz5s3G4/GYUCgUG7N27Vrj8/lMR0dHr75/OBw2kkw4HO7P8gEACdbb5/F+vZIqLy9XWVmZSkpK4rbX1NSoq6srbvvkyZNVWFio6upqSVJ1dbWmTp2qvLy82JjS0lJFIhEdPHjwvN+vo6NDkUgk7gYAGP6S+3qH9evXa8+ePdq9e/fn9oVCIaWmpiozMzNue15enkKhUGzMZwN1dv/ZfeezZs0aPfjgg31dKgBgiOvTK6mGhgbddddd+q//+i+NGjVqoNb0OatWrVI4HI7dGhoaBu17AwASp0+RqqmpUVNTk6644golJycrOTlZVVVVeuyxx5ScnKy8vDx1dnaqpaUl7n6NjY0KBAKSpEAg8Lmz/c5+fXbMubxer3w+X9wNADD89SlS8+bNU21trfbt2xe7zZw5U4sWLYr9d0pKiiorK2P3qaurU319vYLBoCQpGAyqtrZWTU1NsTEVFRXy+XwqLi526WEBAIaDPn0mlZGRoSlTpsRtGzNmjHJycmLbly5dqpUrVyo7O1s+n0933nmngsGg5s6dK0maP3++iouLddttt+mRRx5RKBTSfffdp/Lycnm9XpceFgBgOOjziRNf5tFHH5XH49HChQvV0dGh0tJS/epXv4rtT0pK0qZNm7Rs2TIFg0GNGTNGixcv1kMPPeT2UgAAQ5xjjDGJXkRfRSIR+f1+hcNhPp8CgCGot8/jXLsPAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGCt5EQvoD+MMZKkSCSS4JUAAPrj7PP32efzCxmSkTpx4oQkqaCgIMErAQBcjNbWVvn9/gvuH5KRys7OliTV19d/4YMb6SKRiAoKCtTQ0CCfz5fo5ViL49Q7HKfe4Tj1jjFGra2tys/P/8JxQzJSHs+Zj9L8fj9/CHrB5/NxnHqB49Q7HKfe4Th9ud68yODECQCAtYgUAMBaQzJSXq9XDzzwgLxeb6KXYjWOU+9wnHqH49Q7HCd3OebLzv8DACBBhuQrKQDAyECkAADWIlIAAGsRKQCAtYZkpB5//HFNnDhRo0aN0pw5c7Rr165EL2lQvfbaa/rOd76j/Px8OY6jF198MW6/MUarV6/W+PHjNXr0aJWUlOi9996LG9Pc3KxFixbJ5/MpMzNTS5cuVVtb2yA+ioG1Zs0azZo1SxkZGcrNzdVNN92kurq6uDHt7e0qLy9XTk6O0tPTtXDhQjU2NsaNqa+vV1lZmdLS0pSbm6t77rlH3d3dg/lQBtTatWs1bdq02A+eBoNBbdmyJbafY3R+Dz/8sBzH0YoVK2LbOFYDxAwx69evN6mpqeY///M/zcGDB83tt99uMjMzTWNjY6KXNmg2b95s/vEf/9G88MILRpLZuHFj3P6HH37Y+P1+8+KLL5r9+/ebG2+80RQVFZnTp0/Hxlx33XVm+vTp5s033zSvv/66mTRpkrn11lsH+ZEMnNLSUvOb3/zGHDhwwOzbt8/89V//tSksLDRtbW2xMXfccYcpKCgwlZWV5q233jJz5841X//612P7u7u7zZQpU0xJSYnZu3ev2bx5sxk7dqxZtWpVIh7SgPj9739v/vu//9u8++67pq6uzvzDP/yDSUlJMQcOHDDGcIzOZ9euXWbixIlm2rRp5q677opt51gNjCEXqdmzZ5vy8vLY1z09PSY/P9+sWbMmgatKnHMjFY1GTSAQMD//+c9j21paWozX6zXPPvusMcaYt99+20gyu3fvjo3ZsmWLcRzHHDlyZNDWPpiampqMJFNVVWWMOXNMUlJSzIYNG2JjDh06ZCSZ6upqY8yZfwx4PB4TCoViY9auXWt8Pp/p6OgY3AcwiLKyssyvf/1rjtF5tLa2mksvvdRUVFSYb37zm7FIcawGzpB6u6+zs1M1NTUqKSmJbfN4PCopKVF1dXUCV2aPw4cPKxQKxR0jv9+vOXPmxI5RdXW1MjMzNXPmzNiYkpISeTwe7dy5c9DXPBjC4bCk/704cU1Njbq6uuKO0+TJk1VYWBh3nKZOnaq8vLzYmNLSUkUiER08eHAQVz84enp6tH79ep08eVLBYJBjdB7l5eUqKyuLOyYSf54G0pC6wOzx48fV09MT9z9ZkvLy8vTOO+8kaFV2CYVCknTeY3R2XygUUm5ubtz+5ORkZWdnx8YMJ9FoVCtWrNCVV16pKVOmSDpzDFJTU5WZmRk39tzjdL7jeHbfcFFbW6tgMKj29nalp6dr48aNKi4u1r59+zhGn7F+/Xrt2bNHu3fv/tw+/jwNnCEVKaA/ysvLdeDAAf3xj39M9FKs9NWvflX79u1TOBzW888/r8WLF6uqqirRy7JKQ0OD7rrrLlVUVGjUqFGJXs6IMqTe7hs7dqySkpI+d8ZMY2OjAoFAglZll7PH4YuOUSAQUFNTU9z+7u5uNTc3D7vjuHz5cm3atEnbtm3ThAkTYtsDgYA6OzvV0tISN/7c43S+43h233CRmpqqSZMmacaMGVqzZo2mT5+uX/7ylxyjz6ipqVFTU5OuuOIKJScnKzk5WVVVVXrssceUnJysvLw8jtUAGVKRSk1N1YwZM1RZWRnbFo1GVVlZqWAwmMCV2aOoqEiBQCDuGEUiEe3cuTN2jILBoFpaWlRTUxMbs3XrVkWjUc2ZM2fQ1zwQjDFavny5Nm7cqK1bt6qoqChu/4wZM5SSkhJ3nOrq6lRfXx93nGpra+OCXlFRIZ/Pp+Li4sF5IAkQjUbV0dHBMfqMefPmqba2Vvv27YvdZs6cqUWLFsX+m2M1QBJ95kZfrV+/3ni9XrNu3Trz9ttvmx//+McmMzMz7oyZ4a61tdXs3bvX7N2710gy//Iv/2L27t1rPvroI2PMmVPQMzMzzUsvvWT+9Kc/mQULFpz3FPTLL7/c7Ny50/zxj380l1566bA6BX3ZsmXG7/eb7du3m2PHjsVup06dio254447TGFhodm6dat56623TDAYNMFgMLb/7CnD8+fPN/v27TN/+MMfzLhx44bVKcP33nuvqaqqMocPHzZ/+tOfzL333mscxzH/8z//Y4zhGH2Rz57dZwzHaqAMuUgZY8y//uu/msLCQpOammpmz55t3nzzzUQvaVBt27bNSPrcbfHixcaYM6eh33///SYvL894vV4zb948U1dXFzfHiRMnzK233mrS09ONz+czS5YsMa2trQl4NAPjfMdHkvnNb34TG3P69Gnz93//9yYrK8ukpaWZ7373u+bYsWNx83z44Yfm+uuvN6NHjzZjx441P/nJT0xXV9cgP5qB88Mf/tBccsklJjU11YwbN87MmzcvFihjOEZf5NxIcawGBr+qAwBgrSH1mRQAYGQhUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFr/Hx4xYqb9WdQnAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=16, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=128, out_features=16, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 搭建环境\n",
    "import torch\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(3,128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128,16),\n",
    ")\n",
    "#经验网络,用于评估一个状态的分数\n",
    "next_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(3, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 16),\n",
    ")\n",
    "#把model的参数复制给next_model\n",
    "next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "model, next_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13, 1.4666666666666668)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1,3)\n",
    "    action = model(state).argmax().item()\n",
    "\n",
    "    if random.random() < 0.01:\n",
    "        action = random.choice(range(16))\n",
    "    #离散动作连续化\n",
    "    action_continuous = action\n",
    "    action_continuous /=15\n",
    "    action_continuous *= 4\n",
    "    action_continuous -=2\n",
    "    return action,action_continuous\n",
    "get_action([0,0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((200, 0), 200)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action, action_continuous = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step([action_continuous])\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    update_count = len(datas) - old_count\n",
    "    drop_count = max(len(datas) - 5000, 0)\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 5000:\n",
    "        datas.pop(0)\n",
    "\n",
    "    return update_count, drop_count\n",
    "\n",
    "\n",
    "update_data(), len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cgq10\\AppData\\Local\\Temp\\ipykernel_22504\\3735624114.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1,3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-0.8770, -0.4805, -3.7483],\n",
       "         [-0.9846,  0.1749,  1.5710],\n",
       "         [-0.9505,  0.3106, -0.3568],\n",
       "         [-0.9869,  0.1616,  5.1456],\n",
       "         [-0.4574, -0.8893, -0.9196],\n",
       "         [-0.7721, -0.6355, -3.4917],\n",
       "         [-0.9937,  0.1124, -1.5905],\n",
       "         [-0.7604,  0.6494,  0.4827],\n",
       "         [-0.9359,  0.3522,  0.4649],\n",
       "         [-0.7491, -0.6625,  2.0511],\n",
       "         [-0.9917,  0.1285, -1.8642],\n",
       "         [-0.9181, -0.3964,  1.8319],\n",
       "         [-0.9324,  0.3613,  2.4881],\n",
       "         [-0.9998, -0.0193, -1.8472],\n",
       "         [-0.9261,  0.3772, -0.4389],\n",
       "         [-0.9947, -0.1030,  2.6811],\n",
       "         [-0.9140, -0.4057, -2.9537],\n",
       "         [-0.4079, -0.9130,  0.5672],\n",
       "         [-0.9649, -0.2627, -3.0380],\n",
       "         [-0.8444, -0.5358, -2.7719],\n",
       "         [-0.9179,  0.3968,  0.0153],\n",
       "         [-0.7872,  0.6167,  4.4575],\n",
       "         [-0.9338,  0.3577,  1.5126],\n",
       "         [-0.9438,  0.3306,  0.0565],\n",
       "         [-0.6033, -0.7975, -0.2622],\n",
       "         [-0.9621,  0.2726, -0.5015],\n",
       "         [-0.9388, -0.3445,  5.0734],\n",
       "         [-0.5519, -0.8339,  2.1052],\n",
       "         [-0.7374, -0.6755,  0.9920],\n",
       "         [-0.6088, -0.7933,  0.8568],\n",
       "         [-0.9339,  0.3576, -0.2403],\n",
       "         [-0.9980,  0.0627, -1.6417],\n",
       "         [-0.9844,  0.1760, -1.2862],\n",
       "         [-0.9870,  0.1608,  1.4448],\n",
       "         [-0.4204, -0.9073,  2.9924],\n",
       "         [-0.6943, -0.7197,  4.2781],\n",
       "         [-0.9322,  0.3619,  0.0200],\n",
       "         [-0.4827, -0.8758,  1.6198],\n",
       "         [-0.8954, -0.4453, -1.3371],\n",
       "         [-0.8817, -0.4718,  1.6746],\n",
       "         [-0.9779, -0.2093, -2.0466],\n",
       "         [-0.9404, -0.3400,  2.1790],\n",
       "         [-0.8756, -0.4831,  2.5701],\n",
       "         [-0.9514, -0.3081, -1.5156],\n",
       "         [-0.9995,  0.0309,  2.5179],\n",
       "         [-0.9871,  0.1603, -2.6394],\n",
       "         [-0.8449, -0.5349,  1.4608],\n",
       "         [-0.2119, -0.9773, -1.2468],\n",
       "         [-0.9442,  0.3294, -0.4671],\n",
       "         [-0.9504,  0.3112,  0.8690],\n",
       "         [-0.7680,  0.6404, -1.2998],\n",
       "         [-0.7404,  0.6721, -0.2421],\n",
       "         [-0.8557, -0.5175,  1.8778],\n",
       "         [-0.9994,  0.0356, -2.1109],\n",
       "         [-0.8324, -0.5541, -0.9834],\n",
       "         [-0.9598,  0.2806, -0.6280],\n",
       "         [-0.7660, -0.6428,  0.5759],\n",
       "         [-0.8130, -0.5823,  2.3478],\n",
       "         [-0.7609, -0.6489, -0.8330],\n",
       "         [-0.9760, -0.2180,  1.9484],\n",
       "         [-0.8114, -0.5845,  1.1997],\n",
       "         [-0.9874, -0.1585, -1.7525],\n",
       "         [-0.7649, -0.6441,  0.0338],\n",
       "         [-0.9440, -0.3300, -1.7617]]),\n",
       " tensor([[13],\n",
       "         [11],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [13],\n",
       "         [13],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [11],\n",
       "         [11],\n",
       "         [13],\n",
       "         [11],\n",
       "         [11],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [11],\n",
       "         [13],\n",
       "         [ 6],\n",
       "         [13],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [ 7],\n",
       "         [11],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [11],\n",
       "         [11],\n",
       "         [11],\n",
       "         [ 4],\n",
       "         [13],\n",
       "         [13],\n",
       "         [11],\n",
       "         [11],\n",
       "         [ 6],\n",
       "         [ 4],\n",
       "         [11],\n",
       "         [13],\n",
       "         [11],\n",
       "         [13],\n",
       "         [11],\n",
       "         [11],\n",
       "         [ 0],\n",
       "         [11],\n",
       "         [13],\n",
       "         [11],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [11],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [11],\n",
       "         [13],\n",
       "         [13],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [11],\n",
       "         [13],\n",
       "         [11],\n",
       "         [11],\n",
       "         [ 4],\n",
       "         [ 6],\n",
       "         [13]]),\n",
       " tensor([[ -8.3784],\n",
       "         [ -9.0439],\n",
       "         [ -7.9985],\n",
       "         [-11.5241],\n",
       "         [ -4.2722],\n",
       "         [ -7.2381],\n",
       "         [ -9.4296],\n",
       "         [ -5.9521],\n",
       "         [ -7.7604],\n",
       "         [ -6.2659],\n",
       "         [ -9.4264],\n",
       "         [ -7.8114],\n",
       "         [ -8.3033],\n",
       "         [-10.0919],\n",
       "         [ -7.6089],\n",
       "         [ -9.9516],\n",
       "         [ -8.2940],\n",
       "         [ -3.9963],\n",
       "         [ -9.1951],\n",
       "         [ -7.4072],\n",
       "         [ -7.4735],\n",
       "         [ -8.1226],\n",
       "         [ -7.9344],\n",
       "         [ -7.8672],\n",
       "         [ -4.9283],\n",
       "         [ -8.2369],\n",
       "         [-10.3575],\n",
       "         [ -5.0902],\n",
       "         [ -5.8593],\n",
       "         [ -5.0265],\n",
       "         [ -7.7123],\n",
       "         [ -9.7509],\n",
       "         [ -8.9566],\n",
       "         [ -9.0904],\n",
       "         [ -4.9151],\n",
       "         [ -7.2977],\n",
       "         [ -7.6811],\n",
       "         [ -4.5667],\n",
       "         [ -7.3637],\n",
       "         [ -7.3054],\n",
       "         [ -9.0102],\n",
       "         [ -8.2861],\n",
       "         [ -7.6176],\n",
       "         [ -8.2335],\n",
       "         [-10.3115],\n",
       "         [ -9.5830],\n",
       "         [ -6.8565],\n",
       "         [ -3.3414],\n",
       "         [ -7.8959],\n",
       "         [ -8.0579],\n",
       "         [ -6.1569],\n",
       "         [ -5.7884],\n",
       "         [ -7.1011],\n",
       "         [-10.0948],\n",
       "         [ -6.6231],\n",
       "         [ -8.2036],\n",
       "         [ -6.0036],\n",
       "         [ -6.9028],\n",
       "         [ -6.0030],\n",
       "         [ -8.9179],\n",
       "         [ -6.4815],\n",
       "         [ -9.2026],\n",
       "         [ -5.9623],\n",
       "         [ -8.1822]]),\n",
       " tensor([[-0.9533, -0.3020, -3.8888],\n",
       "         [-0.9965,  0.0836,  1.8422],\n",
       "         [-0.9464,  0.3231, -0.2639],\n",
       "         [-0.9952, -0.0979,  5.2068],\n",
       "         [-0.5170, -0.8560, -1.3666],\n",
       "         [-0.8770, -0.4805, -3.7483],\n",
       "         [-0.9844,  0.1760, -1.2862],\n",
       "         [-0.7867,  0.6173,  0.8298],\n",
       "         [-0.9504,  0.3112,  0.8690],\n",
       "         [-0.6904, -0.7235,  1.6943],\n",
       "         [-0.9788,  0.2048, -1.5478],\n",
       "         [-0.8817, -0.4718,  1.6746],\n",
       "         [-0.9748,  0.2229,  2.8991],\n",
       "         [-0.9980,  0.0627, -1.6417],\n",
       "         [-0.9204,  0.3909, -0.2960],\n",
       "         [-0.9712, -0.2381,  2.7438],\n",
       "         [-0.9649, -0.2627, -3.0380],\n",
       "         [-0.4160, -0.9094, -0.1776],\n",
       "         [-0.9934, -0.1148, -3.0150],\n",
       "         [-0.9140, -0.4057, -2.9537],\n",
       "         [-0.9213,  0.3888,  0.1729],\n",
       "         [-0.9133,  0.4074,  4.9001],\n",
       "         [-0.9638,  0.2665,  1.9209],\n",
       "         [-0.9465,  0.3228,  0.1644],\n",
       "         [-0.6393, -0.7689, -0.9204],\n",
       "         [-0.9559,  0.2936, -0.4370],\n",
       "         [-0.8312, -0.5559,  4.7550],\n",
       "         [-0.4827, -0.8758,  1.6198],\n",
       "         [-0.7159, -0.6982,  0.6254],\n",
       "         [-0.5928, -0.8054,  0.4018],\n",
       "         [-0.9319,  0.3628, -0.1121],\n",
       "         [-0.9914,  0.1311, -1.3747],\n",
       "         [-0.9751,  0.2218, -0.9342],\n",
       "         [-0.9971,  0.0762,  1.7054],\n",
       "         [-0.3063, -0.9519,  2.4519],\n",
       "         [-0.5510, -0.8345,  3.6783],\n",
       "         [-0.9349,  0.3548,  0.1515],\n",
       "         [-0.4337, -0.9011,  1.1030],\n",
       "         [-0.9253, -0.3792, -1.4511],\n",
       "         [-0.8449, -0.5349,  1.4608],\n",
       "         [-0.9938, -0.1115, -1.9836],\n",
       "         [-0.9004, -0.4350,  2.0641],\n",
       "         [-0.8130, -0.5823,  2.3478],\n",
       "         [-0.9779, -0.2093, -2.0466],\n",
       "         [-0.9947, -0.1030,  2.6811],\n",
       "         [-0.9622,  0.2724, -2.2992],\n",
       "         [-0.8114, -0.5845,  1.1997],\n",
       "         [-0.2970, -0.9549, -1.7598],\n",
       "         [-0.9381,  0.3463, -0.3600],\n",
       "         [-0.9678,  0.2516,  1.2424],\n",
       "         [-0.7485,  0.6631, -0.5995],\n",
       "         [-0.7445,  0.6676,  0.1220],\n",
       "         [-0.8107, -0.5855,  1.6296],\n",
       "         [-0.9917,  0.1285, -1.8642],\n",
       "         [-0.8636, -0.5041, -1.1790],\n",
       "         [-0.9516,  0.3073, -0.5575],\n",
       "         [-0.7649, -0.6441,  0.0338],\n",
       "         [-0.7491, -0.6625,  2.0511],\n",
       "         [-0.7954, -0.6061, -1.0997],\n",
       "         [-0.9505, -0.3107,  1.9250],\n",
       "         [-0.7842, -0.6205,  0.9013],\n",
       "         [-0.9983, -0.0586, -2.0114],\n",
       "         [-0.7811, -0.6244, -0.5093],\n",
       "         [-0.9697, -0.2443, -1.7892]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1,3)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1,1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1,1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1,3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1,1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state, action, reward, next_state, over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.1121],\n",
       "        [ 0.7247],\n",
       "        [ 0.4004],\n",
       "        [ 1.8654],\n",
       "        [ 0.4864],\n",
       "        [ 1.0684],\n",
       "        [ 0.5755],\n",
       "        [ 0.4312],\n",
       "        [ 0.4652],\n",
       "        [ 0.9021],\n",
       "        [ 0.6415],\n",
       "        [ 0.8374],\n",
       "        [ 0.9883],\n",
       "        [ 0.6465],\n",
       "        [ 0.4087],\n",
       "        [ 1.0738],\n",
       "        [ 0.9257],\n",
       "        [ 0.4032],\n",
       "        [ 0.9357],\n",
       "        [ 0.8956],\n",
       "        [ 0.4293],\n",
       "        [ 1.6246],\n",
       "        [ 0.7004],\n",
       "        [ 0.4237],\n",
       "        [ 0.3123],\n",
       "        [ 0.3961],\n",
       "        [ 1.8519],\n",
       "        [ 0.8865],\n",
       "        [ 0.5571],\n",
       "        [ 0.4896],\n",
       "        [ 0.4090],\n",
       "        [ 0.5914],\n",
       "        [ 0.4928],\n",
       "        [ 0.6967],\n",
       "        [ 1.1235],\n",
       "        [ 1.5746],\n",
       "        [ 0.4259],\n",
       "        [ 0.7285],\n",
       "        [ 0.5469],\n",
       "        [ 0.7902],\n",
       "        [ 0.7061],\n",
       "        [ 0.9453],\n",
       "        [ 1.0590],\n",
       "        [-0.1881],\n",
       "        [ 1.0218],\n",
       "        [ 0.8180],\n",
       "        [ 0.7273],\n",
       "        [ 0.6010],\n",
       "        [ 0.4034],\n",
       "        [ 0.5696],\n",
       "        [ 0.4826],\n",
       "        [ 0.4315],\n",
       "        [ 0.8580],\n",
       "        [ 0.7068],\n",
       "        [ 0.4520],\n",
       "        [ 0.3941],\n",
       "        [ 0.4520],\n",
       "        [ 0.9911],\n",
       "        [ 0.4276],\n",
       "        [ 0.8650],\n",
       "        [ 0.6392],\n",
       "        [ 0.3435],\n",
       "        [ 0.3640],\n",
       "        [ 0.6420]], grad_fn=<GatherBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #使用状态计算出动作的logits\n",
    "    #[b, 3] -> [b, 11]\n",
    "    value = model(state)\n",
    "\n",
    "    #根据实际使用的action取出每一个值\n",
    "    #这个值就是模型评估的在该状态下,执行动作的分数\n",
    "    #在执行动作前,显然并不知道会得到的反馈和next_state\n",
    "    #所以这里不能也不需要考虑next_state和reward\n",
    "    #[b, 11] -> [b, 1]\n",
    "    value = value.gather(dim=1, index=action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-7.2672],\n",
       "        [-8.2578],\n",
       "        [-7.6026],\n",
       "        [-9.6695],\n",
       "        [-3.6684],\n",
       "        [-6.1482],\n",
       "        [-8.9467],\n",
       "        [-5.4364],\n",
       "        [-7.2022],\n",
       "        [-5.4868],\n",
       "        [-8.8794],\n",
       "        [-7.0370],\n",
       "        [-7.2007],\n",
       "        [-9.5123],\n",
       "        [-7.2062],\n",
       "        [-8.8767],\n",
       "        [-7.3771],\n",
       "        [-3.6974],\n",
       "        [-8.2925],\n",
       "        [-6.5000],\n",
       "        [-7.0441],\n",
       "        [-6.3830],\n",
       "        [-7.1416],\n",
       "        [-7.4463],\n",
       "        [-4.4629],\n",
       "        [-7.8461],\n",
       "        [-8.6463],\n",
       "        [-4.3762],\n",
       "        [-5.4138],\n",
       "        [-4.6282],\n",
       "        [-7.3011],\n",
       "        [-9.2454],\n",
       "        [-8.5515],\n",
       "        [ 0.7484],\n",
       "        [-3.9894],\n",
       "        [-5.9621],\n",
       "        [-7.2565],\n",
       "        [-4.0291],\n",
       "        [-6.8073],\n",
       "        [-6.5927],\n",
       "        [-8.3387],\n",
       "        [-7.3901],\n",
       "        [-6.6463],\n",
       "        [-7.5416],\n",
       "        [-9.2591],\n",
       "        [-8.8655],\n",
       "        [-6.2302],\n",
       "        [-2.6188],\n",
       "        [-7.5000],\n",
       "        [-7.4246],\n",
       "        [-5.7320],\n",
       "        [-5.3561],\n",
       "        [-6.3402],\n",
       "        [-9.4661],\n",
       "        [-6.1286],\n",
       "        [-7.8123],\n",
       "        [-5.6470],\n",
       "        [-6.0188],\n",
       "        [-5.5169],\n",
       "        [-8.0726],\n",
       "        [-5.9602],\n",
       "        [-8.5276],\n",
       "        [-5.6145],\n",
       "        [-7.5508]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #上面已经把模型认为的状态下执行动作的分数给评估出来了\n",
    "    #下面使用next_state和reward计算真实的分数\n",
    "    #针对一个状态,它到底应该多少分,可以使用以往模型积累的经验评估\n",
    "    #这也是没办法的办法,因为显然没有精确解,这里使用延迟更新的next_model评估\n",
    "\n",
    "    #使用next_state计算下一个状态的分数\n",
    "    #[b, 3] -> [b, 11]\n",
    "    with torch.no_grad():\n",
    "        target = next_model(next_state)\n",
    "\n",
    "    #取所有动作中分数最大的\n",
    "    #[b, 11] -> [b, 1]\n",
    "    target = target.max(dim=1)[0]\n",
    "    target = target.reshape(-1, 1)\n",
    "\n",
    "    #下一个状态的分数乘以一个系数,相当于权重\n",
    "    target = 0.98*target +(1-over)*reward\n",
    "\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1481.9829488472265"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        _, action_continuous = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action_continuous])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play: \n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 0 轮——————————分数：-1398.7351950374932\n",
      "第 20 轮——————————分数：-1324.7252988825517\n",
      "第 40 轮——————————分数：-879.5347684474649\n",
      "第 60 轮——————————分数：-376.8828513426814\n",
      "第 80 轮——————————分数：-252.00350637969308\n",
      "第 100 轮——————————分数：-346.4764366971099\n",
      "第 120 轮——————————分数：-826.2385468657849\n",
      "第 140 轮——————————分数：-400.6955411659136\n",
      "第 160 轮——————————分数：-250.01244223052316\n",
      "第 180 轮——————————分数：-305.37696745016405\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(201):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算一批样本的value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(reward, next_state, over)\n",
    "\n",
    "            #更新参数\n",
    "            loss = loss_fn(value, target)\n",
    "            optimizer.zero_grad()#清空梯度防止累积\n",
    "            loss.backward()#计算梯度\n",
    "            optimizer.step()#更新参数\n",
    "\n",
    "            #把model的参数复制给next_model\n",
    "            if (i + 1) % 50 == 0:\n",
    "                next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(20)]) / 20\n",
    "            print(f'第 {epoch} 轮——————————分数：{test_result}')\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzMUlEQVR4nO3de3gUVZ4+8Lf6mmt37t3EJIILgwKCcguNM4ISicrqzIC76iBmHUbHGBgwrqsowoM7Y1xdZWRFXMcVZ3QcXFREucidsEAIEAhCwICCJEA6AZJ07n2r8/sD6R/h4iSk0306/X6eJ89jqqpPfbvs9EtVnTpHEUIIEBERSUgT7AKIiIiuhCFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0gpaSC1cuBC9e/dGREQEMjMzsXPnzmCVQkREkgpKSH388cfIz8/H3LlzsWfPHgwZMgTZ2dmoqakJRjlERCQpJRgDzGZmZmLEiBF48803AQCqqiI9PR3Tp0/Hs88+G+hyiIhIUrpA79DlcqGkpASzZs3yLdNoNMjKykJRUdFlX+N0OuF0On2/q6qK2tpaJCYmQlGUbq+ZiIj8SwiBxsZGpKamQqO58kW9gIfUmTNn4PV6YbFY2i23WCz45ptvLvuagoICzJs3LxDlERFRAFVWViItLe2K6wMeUldj1qxZyM/P9/3ucDiQkZGByspKmEymIFZGRERXo6GhAenp6YiNjf3R7QIeUklJSdBqtaiurm63vLq6Glar9bKvMRqNMBqNlyw3mUwMKSKiEPb3btkEvHefwWDAsGHDsGHDBt8yVVWxYcMG2Gy2QJdDREQSC8rlvvz8fOTk5GD48OEYOXIk/vjHP6K5uRmPPPJIMMohIiJJBSWk7r//fpw+fRpz5syB3W7HTTfdhK+++uqSzhRERBTegvKcVFc1NDTAbDbD4XDwnhQRUQjq6Pc4x+4jIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpJWp0Nqy5YtuOeee5CamgpFUfD555+3Wy+EwJw5c9CrVy9ERkYiKysLR44cabdNbW0tJk+eDJPJhLi4OEydOhVNTU1deiNERNTzdDqkmpubMWTIECxcuPCy61955RUsWLAAb7/9NoqLixEdHY3s7Gy0tbX5tpk8eTLKysqwbt06rFixAlu2bMFjjz129e+CiIh6JtEFAMSyZct8v6uqKqxWq3j11Vd9y+rr64XRaBR/+9vfhBBCHDx4UAAQu3bt8m2zevVqoSiKOHnyZIf263A4BADhcDi6Uj4REQVJR7/H/XpP6tixY7Db7cjKyvItM5vNyMzMRFFREQCgqKgIcXFxGD58uG+brKwsaDQaFBcXX7Zdp9OJhoaGdj9ERNTz+TWk7HY7AMBisbRbbrFYfOvsdjtSUlLardfpdEhISPBtc7GCggKYzWbfT3p6uj/LJiIiSYVE775Zs2bB4XD4fiorK4NdEhERBYBfQ8pqtQIAqqur2y2vrq72rbNaraipqWm33uPxoLa21rfNxYxGI0wmU7sfIiLq+fwaUn369IHVasWGDRt8yxoaGlBcXAybzQYAsNlsqK+vR0lJiW+bjRs3QlVVZGZm+rMcIiIKcbrOvqCpqQnffvut7/djx46htLQUCQkJyMjIwMyZM/H73/8e/fr1Q58+ffDCCy8gNTUVv/jFLwAAN9xwA+688048+uijePvtt+F2uzFt2jQ88MADSE1N9dsbIyKiHqCz3QY3bdokAFzyk5OTI4Q41w39hRdeEBaLRRiNRjFu3DhRXl7ero2zZ8+KBx98UMTExAiTySQeeeQR0djY6Peui0REJKeOfo8rQggRxIy8Kg0NDTCbzXA4HLw/RUQUgjr6PR4SvfuIiCg8MaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSVqdHQSciuQghAFWF8HohVBUQAlAUKBoNFJ0Oiob/FqXQxZAiCmGq2w1nVRXqd+5Ew+7daD1xAmprK7QxMYi89lrE22yIvekmGFNSoGi1wS6XqNMYUkQhSnW5cGbdOpxZswatx4+fO4P6gaeuDo11dWjavx9Rffsi5d57Ef/Tn0JRlCBWTNR5DCmiEKR6PLAvW4bqzz6D2tp6xe2E14vm8nJU/ulPUF0uJI4dyzMqCim8WE0UYlS3G6dXr0b1J5/8aEBdyFNfj5N/+QvqioogvN5urpDIfxhSRCGm5cgR1CxfDtXp7NTrPHV1sH/8MVxnz3ZTZUT+x5AiCiGq2436Xbvgqqm5qte3Hj+Ohr17EYITclOYYkgRhRBvczOqP/20S22c/POf23WyIJIZQ4oohHgcjmCXQBRQDCmiENJ06FCwSyAKKIYUUQip+eKLYJdAFFAMKaIQ4bfODnygl0IIQ4ooRKitrX55xsk8YgSDikIGQ4ooRDjtdqguV5fbMd10U9eLIQoQhhRRiHCUlPild19EaqofqiEKDIYUUYhw19VBeDxdbkcxGDjQLIUMhhRRCFA9Hgi3u8vtGHv1gjYiwg8VEQUGQ4ooBHibmuCuq+tyO9HXXw9tbKwfKiIKDIYUUQhw19bCeepUl9sxWizQGo1+qIgoMBhSRCHAdeYM2k6c6HI72shIKDpOI0ehgyFFJDkhhF86TEBRAE54SCGGIUUkO1X1y1lURFoaYm+80Q8FEQUOQ4pIcqrbjbrt27vcjjY6GvqEBD9URBQ4DCki2Xm9aP3++y43o4mIgI49+yjEMKSIJKf64fkoAFAUBYqGf/IUWviJJZKc027v+ky6Gg1iBg70T0FEAcSQIpJc8+HDXW5D0elguvlmP1RDFFgMKSLJ1W7a1OUzKUVRYOTAshSCGFJEEvPnRIcajjRBIYghRSQxd10dVKezy+1oo6PBcc8pFDGkiCTWfOgQ3H6YQyr57rsB9uyjEMRPLZHEWr77Dt7Gxi63E5GWxinjKSQxpIgkJYTw2z0pfXy8X9ohCjSGFJGkVJcL3qYmv7XH2XgpFDGkiCTlbW6G68yZLrcTff31HLOPQhZDikhSzpMn0bBnT5fbiezdGzqTyQ8VEQUeQ4pIUkKIrg+HBEBvNkNjMPihIqLAY0gRSUgIAY8fup4DALRaDixLIYufXCIZqap/pucwGqE3m7teD1GQMKSIJCRUFY7du7vcjsFiQeyQIX6oiCg4GFJEMvLTlPHayEgYkpL8UBBRcDCkiCTkaWryS6cJRauFotf7oSKi4GBIEUmorbLSb6NN8CFeCmUMKSIJ1Xz5JeD1dq0RjQZJd93ln4KIgoQhRSQhb3Nzl9tQNBrE9O/vh2qIgochRSQZ1eWC6OpZ1A90cXF+aYcoWBhSRJJxnTkDb2urX9ri/SgKdQwpIsm0HD0Kd11dl9uJvO46TnRIIY+fYCLJOE+c8MtEh4ljx0LRav1QEVHw6IJdAFG4UVUVbW1taG1thdPphNPphNvthtfrhaqqaKyqgqKq0HXxLMh4zTWcjZdCHkOKKACcTidOnjyJw4cP49ixY/juu+9w/PhxVFVVoaamBvX19WhubobT6cTwhATMGTIElsjILu1T0Wh4T4pCHkOKyM/OP4SrqipOnz6N9evXY/PmzThw4ABOnjyJ2tpaOJ3Odg/rXhgmGgBdjZaIjAxOGU89QqdCqqCgAJ999hm++eYbREZGYvTo0fiP//gP9L/gWYy2tjY89dRTWLJkCZxOJ7Kzs/HWW2/BYrH4tqmoqEBubi42bdqEmJgY5OTkoKCgADodM5NCm9frxdmzZ1FWVoZPP/0UK1asQHV1NTweD1RVhUajQXR0NJKSkhAbG4uYmBhYLBZYLBaYTCboWloQXVSE6C7+LUSkpbH7OfUInfpLKCwsRF5eHkaMGAGPx4PnnnsO48ePx8GDBxEdHQ0AePLJJ7Fy5UosXboUZrMZ06ZNw8SJE7Ft2zYA5/6IJ0yYAKvViu3bt6OqqgoPP/ww9Ho9XnrpJf+/Q6IAEELAbrdj/fr1+OKLL7B+/Xo0NDQAADQaDVJSUjBw4EAMGTIEP/nJT9CvXz/07t0bFosFERERUBQFQgg0HTyIw34YWNaQkABtVFSX2yEKNkV0YYCw06dPIyUlBYWFhbj11lvhcDiQnJyMjz76CPfddx8A4JtvvsENN9yAoqIijBo1CqtXr8Y//uM/4tSpU76zq7fffhvPPPMMTp8+DUMHZhBtaGiA2WyGw+GAidNiU5Cc/9NpaWnBhg0bsGDBApSWlqK+vh4AEBkZiRtvvBH//M//jJEjRyIjIwNJSUkwGo2XvVckhMDZTZtw/I9/7HJtlkmTkJaT0+V2iLpLR7/Hu3RNwfHDzKEJCQkAgJKSErjdbmRlZfm2uf7665GRkeELqaKiItx4443tLv9lZ2cjNzcXZWVluPnmmy/Zz/keUBe+OaJgc7vdKCkpwYIFC7By5Uo0NzdDq9UiPT0dY8aMwZQpUzBq1ChERERA80NPvR/tyCAEGvft63Jdik4HXWxsl9shksFVh5Sqqpg5cyZuueUWDBo0CABgt9thMBgQd9G1cIvFArvd7tvmwoA6v/78usspKCjAvHnzrrZUIr8SQuDMmTNYsmQJ3nzzTXz77bcAzn2O7733Xtx333342c9+BqPR2LmGVRX1RUVdrk8fH4/YG2/scjtEMrjqkMrLy8OBAwewdetWf9ZzWbNmzUJ+fr7v94aGBqSnp3f7fokupqoqjh49irlz52LlypVobGxEREQERo8ejeeeew5Dhw6FyWS6qq7f/pqaQxMRAUNysl/aIgq2qwqpadOmYcWKFdiyZQvS0tJ8y61WK1wuF+rr69udTVVXV8Nqtfq22blzZ7v2qqurfesux2g0dv5fpUR+5nQ6sX37duTn5+Prr78GcO5y9mOPPYYpU6YgPj6+S88luaqr/TPRoV4Pndnc5XaIZNCpR9qFEJg2bRqWLVuGjRs3ok+fPu3WDxs2DHq9Hhs2bPAtKy8vR0VFBWw2GwDAZrNh//79qKmp8W2zbt06mEwmDBgwoCvvhajbuFwuLF26FHl5edi/fz90Oh3Gjx+PRYsWIS8vDwkJCV1+cLb1xAkIVfVLvXyIl3qKTp1J5eXl4aOPPsLy5csRGxvru4dkNpsRGRkJs9mMqVOnIj8/HwkJCTCZTJg+fTpsNhtGjRoFABg/fjwGDBiAKVOm4JVXXoHdbsfs2bORl5fHsyWSjhACbrcbH374IZ5//nnU1NQgNjYWOTk5yM/PR3p6uq9TRFfVbdkC4XZ3rRFFQcKtt/qlHiIZdCqkFi1aBAAYO3Zsu+WLFy/Gv/zLvwAA5s+fD41Gg0mTJrV7mPc8rVaLFStWIDc3FzabDdHR0cjJycGLL77YtXdC1A2am5vxwQcfYPbs2aivr0dycjKeeeYZ5ObmXrEr+dUQQvhlokMAiLn+er+0QySDToVUR27sRkREYOHChVi4cOEVt7n22muxatWqzuyaKODcbjc+/vhjFBQUoL6+HmlpaXj22WfxyCOPICIiwq/78jY3Q3W5ut6QosDYq1fX2yGSBMchIroMr9eLr776Cs8//zxOnz6NXr16oaCgABMnTuyWy9JOux3uH5477CqlAw/EE4UKhhTRRVRVRXFxMZ5++mmcPn0aiYmJmDdvHh544AEoitItnRKa9u+H0w/DIRlTU6FwokPqQRhSRBcQQuD48eP4wx/+gO+++w4xMTHIy8vDAw884LcOEpfbp7/EZWZC0ev91h5RsPGfXEQXaGlpwfz587F+/XpotVo89NBDmDZtGqK6cbBW4fXC09Tkl7Yi0tJ4JkU9Cj/NRD8QQmDlypV477334PF4kJmZiVmzZiExMbFbnztS29rg/OGB9q7SxcYypKhH4aeZCOcC6rvvvsP8+fPR2toKi8WCF154Addcc02379vjcKDxhxEsiKg9hhQRzj0P9c4776C0tBSRkZGYPn06fvrTnwZk38LthueH6T26ImbgQERmZHS9ICKJMKQo7AkhsHfvXnzyySdwuVwYNWoUfvWrX/kmI+zufXv89BCvPjEROs6vRj0MQ4rCntvtxuLFi1FRUYGoqCg88cQTAR1lv/X77/3Sji4mBho/P2RMFGwMKQp7xcXF+PTTTyGEwF133YV77rmn27qbX0IIVH38sX/aUhR2mqAeh59oCmutra1YvHgxmpubkZiYiN/+9rfQ6QL7+KDwervchiYiApHXXuuHaojkwpCisCWEQGlpqW/iznHjxuGmm24KaA2q0+mXOaS0MTGI/WGGbKKehCFFYcvpdGL16tWoqKhAQkICJk6c6Jd5oTqj7cQJ/5xJGQwwpKT4oSIiuTCkKGzV1NSgsLAQbrcb/fv3h81mC/hkgY5du/w2+rmGA8tSD8SQorAkhMDhw4dRWloKrVaLO+64A6mpqQGvo/X4ccBPZ1JEPRFDisKSqqpYsWIFmpqaEBMTg0mTJgWuR98PhMfjt+nirZMm+aUdItkwpCgsuVwubNy4EQAwaNAg9OnTJ+A1uBsa4PXXwLIBGGni2LFj8Hg83b4fogsxpCgsHTp0CMePHwcAZGdnd8tEhn+P68wZuP0wHBIA6M1mv7RzJS6XC2+88Qa+++67bt0P0cUYUhSWtm7dCqfTCZPJhMzMzIBf6gOA1u++g/PkSf801s0dPg4cOID169dj+fLlUP10iZKoIxhSFHZUVcXevXvh8XiQnp4ekJHOLyaE8Nv9qNjBg7u144TX68X27dtRWVmJzZs3w+Gnae6JOoIhRWGntrYW33//PYQQ6Nu3b7fPF3U5wu2G68wZv7QVO2RIt4bUmTNnsGHDBjQ1NeGbb75BaWmpX2cTJvoxDCkKO6dOncKZM2egKAr69OkDUxBGDve2tKBp/36/tGVISgK0Wr+0dTEhBL799lts27YNAFBVVYXt27fD7XZ3y/6ILsaQorBTU1ODxsZGGI1GXHPNNTAE4Rkjb2srmg8f9ktbikbTbWeCQgisWbMGtbW1AM51oNixYwfq6up4NkUBwZCisHPmzBk0NjYiMjISvXr1CvylPiH88gAvAGiMRmi6sWdiY2MjlixZ0i6QiouL2cuPAoYhRWGnvr4eLS0tiIiIQFJSUsBDCgCcNTVXXHeyuRkrKivxt6NHsf7UKTT/yKW1mAEDEPWTn3RHiQCAtWvXorKyst2y2tparFu3jmdSFBCBnZOASAKtra1wuVwwGo2Ii4sLTg2XmehQCIFjTU2Yu3cvvm9qQpvXC5Nej0Hx8fjPESOgv0w3eZ3Z3G2z8ba1tWH58uWX3H8SQmDlypX4t3/7N0RGRnbLvonO45kUhZ3zZwB6vR6xsbFBqeH0qlWXLDva1IRHt23DIYcDrV4vBACH241tNTWYUVyMs21tl7xG0emgdEOnCSEE9u3bh927d1/2jKm8vBzFxcV+3y/RxRhSFLa0Wm1QRpoAzo3bd7E/lpXBcYVLezvPnMG6U6cuu647Lld6PB5s27btkkt957W2tmLlypXw+uneGtGVMKQobCmKAm03dd3+e9Ife6zLbWijohA3apQfqmlPCIG6ujps3LgRbW1tMBqNvjNOjUYDnU4Hr9eL4uJinLpCcBL5C0OKwpYQIihnAoqiINIPA9pqIiIQ3b+/Hyq6VHl5ObZt24ZevXph1qxZuO+++3zPlT3++ONIT0/H4cOHUVJSwg4U1K0YUhS2VFWFyx8TDvrJhPR06K9w6a53TAwGJyS0W6Zotd3SaUIIgS+//BL9+vXDa6+9hunTp8NoNEIIAZPJhOnTp+O1116DxWLBjh070HaZe2VE/sKQorDldrvR3NwclH3rzWYk3HZbu2XZqamYe/PNiNBqfX+YWkVBotGI10aMwICLeiJaumkOqfr6eng8HrzzzjuYOHEiFEVBa2srAMBgMCAuLg4///nPsXDhQiiKgoaGhm6pgwhgF3QKQwaDATqdDk6nE/V+miqjszSRkYi32eDYvRvexkYA5y4DZqemIi0qCitOnMDZtjb0jonB/X36IPGiDh7GXr0Q303T3SuKgn//939HVFQUFEWBy+XyHafY2FgYjUZotVrccsst6NOnD8zdPE0IhTeGFIUds9mMyMhIOJ1O33A/gaYoCkw334zku+5C9aefQvxwb0xRFAyKj8eg+PgrvlYXH4/UKVO67fmo+Iv23dbWhtraWiiKgri4OBiNRl84BmMEeQovvNxHYScxMRExMTFobW1FdXV10OZH0hiNsPzyl0i4/XYouo79e1EbE4Ne99+PuJEju+X5qMtpaWlBdXU1FEWBxWKBXq8PyH6JAJ5JURiyWCwwmUyoqanBiRMn4Ha7g/a8lDYqCmm//jX0CQmo3bQJrisMl6RotYhITz8XamPHBmwoJyEEHA4HTp06BY1Gg379+gVlGCkKXwwpCjtWqxXx8fHwer2orKxEc3Nz0EJKURTooqPR6777YBo8GHXbt6OprAxOux2q0wltTAwi0tJgHj4c5uHDEZmREfCQOHbsGJqbm6HT6dC/m7q8E10JQ4rCjsVi8d1L+fbbb1FXV4f4+PigniFojEbEDBqE6J/8BN7WVgiPB0JVoWi10Oj10ERFQdPBS4L+pKqqb2ik6Oho3HDDDQGvgcIb70lR2NFqtRg6dCh0Oh2OHj2KqqqqYJcE4NxZlcZohD4uDoakJBhTUmBITITOZApKQAHnLvedn/CwX79+SEpKCkodFL4YUhR2FEVBZmYm9Ho96urqcODAAY6acAVVVVUoLy8HAIwcOZKdJijgGFIUloYOHYqUlBSoqoq1a9fCc5kBX8OdEAIbN25ES0sLDAYDxo4dG7SxDil8MaQoLMXExOC2H0Z82LFjB+x2e5Arko/L5cK6devgdrvxD//wD+w0QUHBkKKwpNFocOedd8JgMKC2thabNm3iJb8LCCHw7bff+i6FDhs2DKmpqex+TgHHkKKwNXToUAwcOBAejweffvopx6C7gKqq2LJlC44ePQqTyYSxY8dy+CMKCoYUhSVFUWC1WjFmzBhoNBrs27cPO3fu5NkU/v98UmvXrkVzczMyMjJw6623QnOZ6euJuhs/dRS2oqKicNtttyE5ORl2ux1r1qxBS0sLgwpAWVkZCgsLoSgKbr/9dvTu3TvYJVGYYkhR2FIUBT/72c8wYsQIeDwefPbZZ77u1uHM6XTizTffhMPhgMlkwmOPPcZefRQ0DCkKa2azGQ899BCMRiMqKirwwQcfBG3AWRkIIbB161Zs3LgRAHDvvffiuuuuC3JVFM4YUhT27rjjDmRmZkIIgSVLlmDfvn1heclPCIGzZ8/iT3/6ExoaGmC1WjFlyhQYDIZgl0ZhjCFFYS82NhZPPvkkEhIScPr0abz88suoq6sLdlkB5/F4sGzZMqxfvx4ajQYTJ07EiBEj2O2cgoohRWFPo9Fg9OjRuPPOO6EoCjZt2oQvv/wyrEahEELg+PHjeOedd1BfX4+0tDQ89NBDiI2NDXZpFOYYUkQAEhIS8OijjyI9PR21tbV44403cPTo0bC57Od2uzF//nyUlpZCq9XikUcewbBhw3gWRUHHkCLCubOpn/70p8jLy4Ner8e+ffswd+5cOByOYJfW7VwuFz744AN8+OGHEEJg7NixeOKJJ6AL0sjrRBdiSBH9QKPRICcnB1lZWQCAFStW4P3330dbW1uQK+s+qqpi586dWLBgAZqamtC7d2/k5+cjLi4u2KURAWBIEbWTmJiI2bNnY9CgQWhtbcX8+fOxfPlyuN3uYJfmd0IInDhxAs8//zwOHjyI2NhY5ObmYuzYsRxdgqTBTyLRBRRFwYgRIzB79mxYLBacOHECs2bNwsaNG3vU81NCCJw6dQpPPfUUtm7dCkVR8Ktf/Qq5ubkwGo3BLo/IhyFFdBGtVot77rkHv/vd76DX63H8+HE899xz2LFjB1RVDfnOFEII1NTU4OWXX8bKlSuhKAqysrLwzDPPICoqip0lSCoMKaLLiIiIQG5uLvLy8hAVFYV9+/ZhxowZ2Lx5c0ifUQkhcPr0abzwwgt477334HK5MGbMGLz00ktIT08PdnlEl2BIEV2ByWTCM888g0cffRQRERHYs2cPnnjiCaxatSok71GpqoojR47gX//1X/Hee++hra0No0aNwuuvv44hQ4bwPhRJiZ9Koh+RnJyMZ555Bjk5OdBqtThy5Ajy8/Px4YcfhtSI6aqqYs+ePcjPz8fSpUsBACNHjsR//ud/YvDgwbzER9JiSBH9CEVRkJKSgt///vd46qmnEB8fj6NHj+Kpp57CnDlzcOrUKakv/wkh0NjYiGXLluGBBx7AmjVroKoqsrOzsWjRIowcOTLYJRL9KEWEyj8FL9DQ0ACz2eybSoAoEBoaGvDXv/4VBQUFOHnyJBRFwfjx4zFjxgzceuutiIiIkOqMRAiBQ4cO4d1338Vf/vIX1NXVITIyEpMmTcK8efM4RxQFVUe/xxlSRJ3gdDqxevVqzJ07F2VlZVAUBRkZGbj//vvxu9/9DikpKVAUJWhhdf7PuaWlBatWrcLrr7+O0tJSuFwuJCUl4be//S1yc3NhtVqlClQKPx39Hu/U5b5FixZh8ODBMJlMMJlMsNlsWL16tW99W1sb8vLykJiYiJiYGEyaNAnV1dXt2qioqMCECRMQFRWFlJQUPP3002E1kCeFNoPBgHvvvRf/+7//i5ycHJhMJnz//fd47bXXcNddd+F//ud/UFFRAa/XG/DaVFVFfX091q9fj8mTJ+M3v/kNdu7cCQAYNmwY3n//fTz//PMMKAopnTqT+vLLL6HVatGvXz8IIfDnP/8Zr776Kvbu3YuBAwciNzcXK1euxPvvvw+z2Yxp06ZBo9Fg27ZtAACv14ubbroJVqsVr776KqqqqvDwww/j0UcfxUsvvdThonkmRTJoamrCRx99hPfeew979+6Fx+OB0WhEZmYm/umf/gm33347+vbt2+1j4J1/7qmwsBCrV6/GihUrUFtbCwBIS0vD/fffj9/85jfo168fw4mk0eHvcdFF8fHx4t133xX19fVCr9eLpUuX+tYdOnRIABBFRUVCCCFWrVolNBqNsNvtvm0WLVokTCaTcDqdHd6nw+EQAITD4ehq+URXTVVV4XK5xKFDh8TcuXNFr169hE6nExqNRsTGxoohQ4aIvLw8UVhYKOrq6oTL5RJer1eoqtqlfaqqKjwej2htbRVlZWXitddeE2PGjBHJyclCq9UKrVYrYmNjxT333CM2b94sGhsbu7RPou7Q0e/xq74n5fV6sXTpUuTk5GDv3r2w2+0YN24c6urq2g1Oee2112LmzJl48sknMWfOHHzxxRcoLS31rT927Biuu+467NmzBzfffPNl9+V0OuF0OtslcHp6Os+kSApCCHi9XpSXl2Px4sVYu3Ytjhw5ApfLBUVREBERgT59+uD222/HrbfeimuvvRbJyckwm82IiYmBVqv90TMcVVXR0tKCxsZG1NXV4cSJE/j666+xdu1alJSUoLGxEV6vFxqNBklJScjMzMTUqVNx2223ITo6mmdPJKWOnkl1+jrE/v37YbPZ0NbWhpiYGCxbtgwDBgxAaWkpDAbDJaMnWywW2O12AIDdbofFYrlk/fl1V1JQUIB58+Z1tlSigFAUBTqdDgMHDkRBQQGmTJmClStXYv369di9ezeam5tx8OBBHDx4EO+88w4sFguuu+46pKenw2q1IiUlBfHx8YiOjvaNm+fxeNDc3IyGhgacPXsWVVVVOHXqFI4fP47jx4+jqanJt3+NRoP+/fvj9ttvR1ZWFsaOHctRzKnH6HRI9e/fH6WlpXA4HPjkk0+Qk5ODwsLC7qjNZ9asWcjPz/f9fv5Mikg2er0egwcPxg033ICHH34Yhw4dwvLly1FYWIiKigo4nU6cOHEClZWVAM4FnMFggMFggE6ng1arBXDu7Mnj8cDtdsPpdLZ7Fkur1SIiIgIJCQkYNWoU7r77bthsNqSlpfHMiXqcToeUwWBA3759AZzrMbRr1y688cYbuP/+++FyuVBfX9/uX3HV1dWwWq0AAKvV6uttdOH68+uuxGg0cmRmChnng+eaa67BNddcg3HjxqGxsRFlZWXYtm0bDhw4gIqKCpw9exaNjY1obm5GW1sbmpqa4PV6IYSARqOBXq+HwWCA2WyGyWRCXFwcLBYLBg4ciBEjRmD48OHo1auXbzgjhhP1RF3udqSqKpxOJ4YNGwa9Xo8NGzZg0qRJAIDy8nJUVFTAZrMBAGw2G/7whz+gpqYGKSkpAIB169bBZDJhwIABXS2FSCrnQ0NRFJjNZowePRqjR4+G2+3G2bNnUVNTg7Nnz6Kurs4XVG632xdSRqMRUVFRMJvNSExMRHJyMpKTk3m2RGGlUyE1a9Ys3HXXXcjIyEBjYyM++ugjbN68GWvWrIHZbMbUqVORn5+PhIQEmEwmTJ8+HTabDaNGjQIAjB8/HgMGDMCUKVPwyiuvwG63Y/bs2cjLy+OZEoUNvV4Pq9X6o1cPiOicToVUTU0NHn74YVRVVcFsNmPw4MFYs2YN7rjjDgDA/PnzodFoMGnSJDidTmRnZ+Ott97yvV6r1WLFihXIzc2FzWZDdHQ0cnJy8OKLL/r3XRERUY/AYZGIiCjgumVYJCIiokBiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJK0uhdTLL78MRVEwc+ZM37K2tjbk5eUhMTERMTExmDRpEqqrq9u9rqKiAhMmTEBUVBRSUlLw9NNPw+PxdKUUIiLqga46pHbt2oX//u//xuDBg9stf/LJJ/Hll19i6dKlKCwsxKlTpzBx4kTfeq/XiwkTJsDlcmH79u3485//jPfffx9z5sy5+ndBREQ9k7gKjY2Nol+/fmLdunVizJgxYsaMGUIIIerr64VerxdLly71bXvo0CEBQBQVFQkhhFi1apXQaDTCbrf7tlm0aJEwmUzC6XR2aP8Oh0MAEA6H42rKJyKiIOvo9/hVnUnl5eVhwoQJyMrKare8pKQEbre73fLrr78eGRkZKCoqAgAUFRXhxhtvhMVi8W2TnZ2NhoYGlJWVXXZ/TqcTDQ0N7X6IiKjn03X2BUuWLMGePXuwa9euS9bZ7XYYDAbExcW1W26xWGC3233bXBhQ59efX3c5BQUFmDdvXmdLJSKiENepM6nKykrMmDEDf/3rXxEREdFdNV1i1qxZcDgcvp/KysqA7ZuIiIKnUyFVUlKCmpoaDB06FDqdDjqdDoWFhViwYAF0Oh0sFgtcLhfq6+vbva66uhpWqxUAYLVaL+ntd/7389tczGg0wmQytfshIqKer1MhNW7cOOzfvx+lpaW+n+HDh2Py5Mm+/9br9diwYYPvNeXl5aioqIDNZgMA2Gw27N+/HzU1Nb5t1q1bB5PJhAEDBvjpbRERUU/QqXtSsbGxGDRoULtl0dHRSExM9C2fOnUq8vPzkZCQAJPJhOnTp8Nms2HUqFEAgPHjx2PAgAGYMmUKXnnlFdjtdsyePRt5eXkwGo1+eltERNQTdLrjxN8zf/58aDQaTJo0CU6nE9nZ2Xjrrbd867VaLVasWIHc3FzYbDZER0cjJycHL774or9LISKiEKcIIUSwi+ishoYGmM1mOBwO3p8iIgpBHf0e59h9REQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQtXbALuBpCCABAQ0NDkCshIqKrcf77+/z3+ZWEZEidPXsWAJCenh7kSoiIqCsaGxthNpuvuD4kQyohIQEAUFFR8aNvLtw1NDQgPT0dlZWVMJlMwS5HWjxOHcPj1DE8Th0jhEBjYyNSU1N/dLuQDCmN5tytNLPZzA9BB5hMJh6nDuBx6hgep47hcfr7OnKSwY4TREQkLYYUERFJKyRDymg0Yu7cuTAajcEuRWo8Th3D49QxPE4dw+PkX4r4e/3/iIiIgiQkz6SIiCg8MKSIiEhaDCkiIpIWQ4qIiKQVkiG1cOFC9O7dGxEREcjMzMTOnTuDXVJAbdmyBffccw9SU1OhKAo+//zzduuFEJgzZw569eqFyMhIZGVl4ciRI+22qa2txeTJk2EymRAXF4epU6eiqakpgO+iexUUFGDEiBGIjY1FSkoKfvGLX6C8vLzdNm1tbcjLy0NiYiJiYmIwadIkVFdXt9umoqICEyZMQFRUFFJSUvD000/D4/EE8q10q0WLFmHw4MG+B09tNhtWr17tW89jdHkvv/wyFEXBzJkzfct4rLqJCDFLliwRBoNBvPfee6KsrEw8+uijIi4uTlRXVwe7tIBZtWqVeP7558Vnn30mAIhly5a1W//yyy8Ls9ksPv/8c7Fv3z5x7733ij59+ojW1lbfNnfeeacYMmSI2LFjh/i///s/0bdvX/Hggw8G+J10n+zsbLF48WJx4MABUVpaKu6++26RkZEhmpqafNs8/vjjIj09XWzYsEHs3r1bjBo1SowePdq33uPxiEGDBomsrCyxd+9esWrVKpGUlCRmzZoVjLfULb744guxcuVKcfjwYVFeXi6ee+45odfrxYEDB4QQPEaXs3PnTtG7d28xePBgMWPGDN9yHqvuEXIhNXLkSJGXl+f73ev1itTUVFFQUBDEqoLn4pBSVVVYrVbx6quv+pbV19cLo9Eo/va3vwkhhDh48KAAIHbt2uXbZvXq1UJRFHHy5MmA1R5INTU1AoAoLCwUQpw7Jnq9XixdutS3zaFDhwQAUVRUJIQ4948BjUYj7Ha7b5tFixYJk8kknE5nYN9AAMXHx4t3332Xx+gyGhsbRb9+/cS6devEmDFjfCHFY9V9Qupyn8vlQklJCbKysnzLNBoNsrKyUFRUFMTK5HHs2DHY7fZ2x8hsNiMzM9N3jIqKihAXF4fhw4f7tsnKyoJGo0FxcXHAaw4Eh8MB4P8PTlxSUgK3293uOF1//fXIyMhod5xuvPFGWCwW3zbZ2dloaGhAWVlZAKsPDK/XiyVLlqC5uRk2m43H6DLy8vIwYcKEdscE4OepO4XUALNnzpyB1+tt9z8ZACwWC7755psgVSUXu90OAJc9RufX2e12pKSktFuv0+mQkJDg26YnUVUVM2fOxC233IJBgwYBOHcMDAYD4uLi2m178XG63HE8v66n2L9/P2w2G9ra2hATE4Nly5ZhwIABKC0t5TG6wJIlS7Bnzx7s2rXrknX8PHWfkAopoquRl5eHAwcOYOvWrcEuRUr9+/dHaWkpHA4HPvnkE+Tk5KCwsDDYZUmlsrISM2bMwLp16xARERHscsJKSF3uS0pKglarvaTHTHV1NaxWa5Cqksv54/Bjx8hqtaKmpqbdeo/Hg9ra2h53HKdNm4YVK1Zg06ZNSEtL8y23Wq1wuVyor69vt/3Fx+lyx/H8up7CYDCgb9++GDZsGAoKCjBkyBC88cYbPEYXKCkpQU1NDYYOHQqdTgedTofCwkIsWLAAOp0OFouFx6qbhFRIGQwGDBs2DBs2bPAtU1UVGzZsgM1mC2Jl8ujTpw+sVmu7Y9TQ0IDi4mLfMbLZbKivr0dJSYlvm40bN0JVVWRmZga85u4ghMC0adOwbNkybNy4EX369Gm3ftiwYdDr9e2OU3l5OSoqKtodp/3797cL9HXr1sFkMmHAgAGBeSNBoKoqnE4nj9EFxo0bh/3796O0tNT3M3z4cEyePNn33zxW3STYPTc6a8mSJcJoNIr3339fHDx4UDz22GMiLi6uXY+Znq6xsVHs3btX7N27VwAQr7/+uti7d684fvy4EOJcF/S4uDixfPly8fXXX4uf//znl+2CfvPNN4vi4mKxdetW0a9fvx7VBT03N1eYzWaxefNmUVVV5ftpaWnxbfP444+LjIwMsXHjRrF7925hs9mEzWbzrT/fZXj8+PGitLRUfPXVVyI5OblHdRl+9tlnRWFhoTh27Jj4+uuvxbPPPisURRFr164VQvAY/ZgLe/cJwWPVXUIupIQQ4r/+679ERkaGMBgMYuTIkWLHjh3BLimgNm3aJABc8pOTkyOEONcN/YUXXhAWi0UYjUYxbtw4UV5e3q6Ns2fPigcffFDExMQIk8kkHnnkEdHY2BiEd9M9Lnd8AIjFixf7tmltbRVPPPGEiI+PF1FRUeKXv/ylqKqqatfO999/L+666y4RGRkpkpKSxFNPPSXcbneA3033+fWvfy2uvfZaYTAYRHJyshg3bpwvoITgMfoxF4cUj1X34FQdREQkrZC6J0VEROGFIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCSt/wcEBBdIvT8J+gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-121.3944744982945"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Gym",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
